Weekly notes text analysis

library(readr)
library(tibble)
library(tidyr)
library(dplyr)
library(tidytext)
library(ggplot2)
library(tm)
library(topicmodels)
library(stringr)
library(koRpus)

weeknotes_ryan <- as_tibble(read_csv('weeknotes_ryan.csv', col_names = FALSE))
names(weeknotes_ryan) <- c( "s02e01",
                            "s01e09",
                            "s01e08",
                            "s01e07",
                            "s01e06",
                            "s01e05",
                            "s01e04",
                            "s01e03",
                            "s01e02",      
                            "s01e01"
       )

weeknotes_ryan %>% 
  gather("episode", "text", 1:10) -> tidy_notes

tidy_notes

tidy_notes %>% 
  mutate(text2 = gsub("//.", " ", text)) -> tidier_notes

tidy_notes %>% 
  mutate(text2 = gsub("[.]", " ", text)) -> tidier_notes

tidier_notes %>% 
  unnest_tokens(word, text2) -> tidy_tokens

tidy_tokens

treetag(tidy_tokens$word,
        treetagger = "manual",
        format = "obj",
        TT.tknz = FALSE,
        lang = "en",
        TT.options = list(path = "../TreeTagger/", preset = "en")) -> tidy_tokens_tagged

Assuming 'UTF-8' as encoding for the input file. If the results turn out to be erroneous, check the file for invalid characters, e.g. em.dashes or fancy quotes, and/or consider setting 'encoding' manually.
Can't find the lexicon file, hence omitted! Please ensure this path is valid:
  ../TreeTagger//lib/english-lexicon.txt

as_tibble(tidy_tokens_tagged@TT.res)

data("stop_words")
other_stop_words <- c("i’d", "i’m", "i’ve")

tidy_tokens_tagged@TT.res %>%
  as_tibble() %>% 
  anti_join(stop_words, by = c("token" = "word")) %>% 
  filter(!(token %in% other_stop_words)) -> tidier_wo_stopwords

tidier_wo_stopwords %>% 
  filter(str_detect(token, regex("i’d", ignore_case = TRUE)))

tidier_wo_stopwords %>% 
  rename(word = token) -> tidier_wo_stopwords

tidier_wo_stopwords

tidier_wo_stopwords %>% 
  count(word, sort=TRUE)

tidier_wo_stopwords %>% 
  filter(episode == 's01e01') %>% 
  count(word, sort=TRUE)

tidier_wo_stopwords %>% 
  filter(episode == 's02e01') %>% 
  count(word, sort=TRUE)

tidier_wo_stopwords %>%
  count(word, sort = TRUE) %>%
  filter(n > 30) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(word, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

tidier_wo_stopwords

tidier_wo_stopwords %>%
  count(episode, word, sort = TRUE) %>%
  ungroup() -> kk

kk %>% 
  bind_tf_idf(word, episode, n) -> tidier_tf_idf 

tidier_tf_idf %>% 
  arrange(desc(tf_idf))

tidier_tf_idf %>%
  arrange(desc(tf_idf)) %>%
  mutate(word = factor(word, levels = rev(unique(word)))) -> plot_episodes_tf_idf

plot_episodes_tf_idf %>% 
  group_by(episode) %>% 
  top_n(7) %>% 
  ungroup %>%
  ggplot(aes(word, tf_idf, fill = episode)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~episode, ncol = 2, scales = "free") +
  coord_flip()

tidier_notes %>% 
  unnest_tokens(bigram, text, token = "ngrams", n = 2) -> bigrams

bigrams %>%
  count(bigram, sort = TRUE)

bigrams_separated <- bigrams %>%
  separate(bigram, c("word1", "word2"), sep = " ")

bigrams_filtered <- bigrams_separated %>%
  filter(!word1 %in% stop_words$word) %>%
  filter(!word2 %in% stop_words$word) %>% 
  filter(!word2 %in% other_stop_words) %>% 
  filter(!word2 %in% other_stop_words)
  

# new bigram counts:
bigram_counts <- bigrams_filtered %>% 
  count(word1, word2, sort = TRUE)

bigram_counts

tidier_notes %>% 
  unnest_tokens(trigram, text, token = "ngrams", n = 3) -> trigrams

trigrams_separated <- trigrams %>%
  separate(trigram, c("word1", "word2", "word3"), sep = " ")

trigrams_filtered <- trigrams_separated %>%
  filter(!word1 %in% stop_words$word) %>%
  filter(!word3 %in% stop_words$word) %>% 
  filter(!word1 %in% other_stop_words) %>% 
  filter(!word3 %in% other_stop_words)
  

# new bigram counts:
trigram_counts <- trigrams_filtered %>% 
  count(word1, word2, word3, sort = TRUE)

trigram_counts

tidier_tf_idf %>% 
  cast_dtm(document = episode,term = word, value = n) -> dtm_matrix

dtm_matrix

tidier_tf_idf %>% 
  cast_sparse(episode, word, n) -> sparse_matrix

str(sparse_matrix)

txt_lda <- LDA(dtm_matrix, k = 5, control = list(seed = 1234))

txt_topics <- tidy(txt_lda, matrix = "beta")
top_terms <- txt_topics %>%
  group_by(topic) %>%
  top_n(10, beta) %>%
  ungroup() %>%
  arrange(topic, -beta)

top_terms %>%
  mutate(term = reorder(term, beta)) %>%
  ggplot(aes(term, beta, fill = factor(topic))) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~ topic, scales = "free") +
  coord_flip() +
  theme_bw()